{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.586337Z",
     "start_time": "2025-01-26T10:18:58.545835Z"
    }
   },
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.1\n",
      "torch 2.5.1+cpu\n",
      "cpu\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.710479Z",
     "start_time": "2025-01-26T10:19:01.587264Z"
    }
   },
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "print(housing.DESCR)\n",
    "print(housing.data.shape)\n",
    "print(housing.target.shape)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".. _california_housing_dataset:\n",
      "\n",
      "California Housing dataset\n",
      "--------------------------\n",
      "\n",
      "**Data Set Characteristics:**\n",
      "\n",
      ":Number of Instances: 20640\n",
      "\n",
      ":Number of Attributes: 8 numeric, predictive attributes and the target\n",
      "\n",
      ":Attribute Information:\n",
      "    - MedInc        median income in block group\n",
      "    - HouseAge      median house age in block group\n",
      "    - AveRooms      average number of rooms per household\n",
      "    - AveBedrms     average number of bedrooms per household\n",
      "    - Population    block group population\n",
      "    - AveOccup      average number of household members\n",
      "    - Latitude      block group latitude\n",
      "    - Longitude     block group longitude\n",
      "\n",
      ":Missing Attribute Values: None\n",
      "\n",
      "This dataset was obtained from the StatLib repository.\n",
      "https://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.html\n",
      "\n",
      "The target variable is the median house value for California districts,\n",
      "expressed in hundreds of thousands of dollars ($100,000).\n",
      "\n",
      "This dataset was derived from the 1990 U.S. census, using one row per census\n",
      "block group. A block group is the smallest geographical unit for which the U.S.\n",
      "Census Bureau publishes sample data (a block group typically has a population\n",
      "of 600 to 3,000 people).\n",
      "\n",
      "A household is a group of people residing within a home. Since the average\n",
      "number of rooms and bedrooms in this dataset are provided per household, these\n",
      "columns may take surprisingly large values for block groups with few households\n",
      "and many empty houses, such as vacation resorts.\n",
      "\n",
      "It can be downloaded/loaded using the\n",
      ":func:`sklearn.datasets.fetch_california_housing` function.\n",
      "\n",
      ".. rubric:: References\n",
      "\n",
      "- Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,\n",
      "  Statistics and Probability Letters, 33 (1997) 291-297\n",
      "\n",
      "(20640, 8)\n",
      "(20640,)\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.716470Z",
     "start_time": "2025-01-26T10:19:01.712474Z"
    }
   },
   "source": [
    "# print(housing.data[0:5])\n",
    "import pprint  #打印的格式比较 好看\n",
    "\n",
    "pprint.pprint(housing.data[0:5])\n",
    "print('-'*50)\n",
    "pprint.pprint(housing.target[0:5])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[ 8.32520000e+00,  4.10000000e+01,  6.98412698e+00,\n",
      "         1.02380952e+00,  3.22000000e+02,  2.55555556e+00,\n",
      "         3.78800000e+01, -1.22230000e+02],\n",
      "       [ 8.30140000e+00,  2.10000000e+01,  6.23813708e+00,\n",
      "         9.71880492e-01,  2.40100000e+03,  2.10984183e+00,\n",
      "         3.78600000e+01, -1.22220000e+02],\n",
      "       [ 7.25740000e+00,  5.20000000e+01,  8.28813559e+00,\n",
      "         1.07344633e+00,  4.96000000e+02,  2.80225989e+00,\n",
      "         3.78500000e+01, -1.22240000e+02],\n",
      "       [ 5.64310000e+00,  5.20000000e+01,  5.81735160e+00,\n",
      "         1.07305936e+00,  5.58000000e+02,  2.54794521e+00,\n",
      "         3.78500000e+01, -1.22250000e+02],\n",
      "       [ 3.84620000e+00,  5.20000000e+01,  6.28185328e+00,\n",
      "         1.08108108e+00,  5.65000000e+02,  2.18146718e+00,\n",
      "         3.78500000e+01, -1.22250000e+02]])\n",
      "--------------------------------------------------\n",
      "array([4.526, 3.585, 3.521, 3.413, 3.422])\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.759578Z",
     "start_time": "2025-01-26T10:19:01.716470Z"
    }
   },
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#拆分训练集和测试集，random_state是随机种子,同样的随机数种子，是为了得到同样的随机值\n",
    "x_train_all, x_test, y_train_all, y_test = train_test_split(\n",
    "    housing.data, housing.target, random_state = 7)\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(\n",
    "    x_train_all, y_train_all, random_state = 11)\n",
    "# 训练集\n",
    "print(x_train.shape, y_train.shape)\n",
    "# 验证集\n",
    "print(x_valid.shape, y_valid.shape)\n",
    "# 测试集\n",
    "print(x_test.shape, y_test.shape)\n",
    "\n",
    "dataset_maps = {\n",
    "    \"train\": [x_train, y_train],\n",
    "    \"valid\": [x_valid, y_valid],\n",
    "    \"test\": [x_test, y_test],\n",
    "}\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11610, 8) (11610,)\n",
      "(3870, 8) (3870,)\n",
      "(5160, 8) (5160,)\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.768307Z",
     "start_time": "2025-01-26T10:19:01.760576Z"
    }
   },
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(x_train)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StandardScaler()"
      ],
      "text/html": [
       "<style>#sk-container-id-1 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: #000;\n",
       "  --sklearn-color-text-muted: #666;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-1 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-1 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: flex;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "  align-items: start;\n",
       "  justify-content: space-between;\n",
       "  gap: 0.5em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label .caption {\n",
       "  font-size: 0.6rem;\n",
       "  font-weight: lighter;\n",
       "  color: var(--sklearn-color-text-muted);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-1 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-1 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 0.5em;\n",
       "  text-align: center;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>StandardScaler()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow\"><div><div>StandardScaler</div></div><div><a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.6/modules/generated/sklearn.preprocessing.StandardScaler.html\">?<span>Documentation for StandardScaler</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></div></label><div class=\"sk-toggleable__content fitted\"><pre>StandardScaler()</pre></div> </div></div></div></div>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 构建数据集"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.777298Z",
     "start_time": "2025-01-26T10:19:01.769298Z"
    }
   },
   "source": [
    "from torch.utils.data import Dataset\n",
    "\n",
    "class HousingDataset(Dataset):\n",
    "    def __init__(self, mode='train'):\n",
    "        self.x, self.y = dataset_maps[mode]\n",
    "        self.x = torch.from_numpy(scaler.transform(self.x)).float()\n",
    "        self.y = torch.from_numpy(self.y).float().reshape(-1, 1)\n",
    "            \n",
    "    def __len__(self):\n",
    "        return len(self.x)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return self.x[idx], self.y[idx]\n",
    "    \n",
    "    \n",
    "train_ds = HousingDataset(\"train\")\n",
    "valid_ds = HousingDataset(\"valid\")\n",
    "test_ds = HousingDataset(\"test\")"
   ],
   "outputs": [],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.787073Z",
     "start_time": "2025-01-26T10:19:01.778298Z"
    }
   },
   "source": [
    "train_ds[1]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([-0.2981,  0.3523, -0.1092, -0.2506, -0.0341, -0.0060,  1.0806, -1.0611]),\n",
       " tensor([1.5140]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.794208Z",
     "start_time": "2025-01-26T10:19:01.787925Z"
    }
   },
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "batch_size = 8\n",
    "train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=False)\n",
    "val_loader = DataLoader(valid_ds, batch_size=batch_size, shuffle=False)\n",
    "test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False)"
   ],
   "outputs": [],
   "execution_count": 8
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型\n",
    "\n",
    "多输出模型常见于多任务学习（Multi-Task Learning）中，一个任务会有一个输出层，计算一个loss，最后多个任务的loss联合训练模型。除此之外，如果想拿到模型的中间输出，也可以使用多输出。\n",
    "\n",
    "例如，这里想拿到 deep_output，我们构建多输出模型"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.804442Z",
     "start_time": "2025-01-26T10:19:01.795242Z"
    }
   },
   "source": [
    "#回归模型我们只需要1个数\n",
    "\n",
    "class WideDeep(nn.Module):\n",
    "    def __init__(self, input_dim=8):\n",
    "        super().__init__()\n",
    "        self.deep = nn.Sequential(\n",
    "            nn.Linear(input_dim, 30),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(30, 30),\n",
    "            nn.ReLU()\n",
    "            )\n",
    "        # pytorch 需要自行计算输出输出维度\n",
    "        self.output_layer = nn.Linear(30 + input_dim, 1)\n",
    "        \n",
    "        # 初始化权重\n",
    "        self.init_weights()\n",
    "        \n",
    "    def init_weights(self):\n",
    "        \"\"\"使用 xavier 均匀分布来初始化全连接层的权重 W\"\"\"\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Linear):\n",
    "                nn.init.xavier_uniform_(m.weight)\n",
    "                nn.init.zeros_(m.bias)\n",
    "        \n",
    "    def forward(self, x, return_deep_output=False):\n",
    "        # x.shape [batch size, 8]\n",
    "        deep_output = self.deep(x)\n",
    "        # concat [batch size, 30] with [batch size 8]\n",
    "        concat = torch.cat([x, deep_output], dim=-1)\n",
    "        logits = self.output_layer(concat)\n",
    "        # logits.shape [batch size, 1]\n",
    "        return (logits, deep_output) if return_deep_output else logits #如果return_deep_output为True，返回logits和deep_output，否则只返回logits"
   ],
   "outputs": [],
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.814240Z",
     "start_time": "2025-01-26T10:19:01.804442Z"
    }
   },
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        \"\"\"\n",
    "\n",
    "        Args:\n",
    "            patience (int, optional): Number of epochs with no improvement after which training will be stopped.. Defaults to 5.\n",
    "            min_delta (float, optional): Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute \n",
    "                change of less than min_delta, will count as no improvement. Defaults to 0.01.\n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.min_delta = min_delta\n",
    "        self.best_metric = -1\n",
    "        self.counter = 0\n",
    "        \n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:\n",
    "            # update best metric\n",
    "            self.best_metric = metric\n",
    "            # reset counter \n",
    "            self.counter = 0\n",
    "        else: \n",
    "            self.counter += 1\n",
    "            \n",
    "    @property\n",
    "    def early_stop(self):\n",
    "        return self.counter >= self.patience\n"
   ],
   "outputs": [],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:01.823678Z",
     "start_time": "2025-01-26T10:19:01.815237Z"
    }
   },
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "@torch.no_grad()\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    for datas, labels in dataloader:\n",
    "        datas = datas.to(device)\n",
    "        labels = labels.to(device)\n",
    "        # 前向计算\n",
    "        logits = model(datas)\n",
    "        loss = loss_fct(logits, labels)         # 验证集损失\n",
    "        loss_list.append(loss.item())\n",
    "        \n",
    "    return np.mean(loss_list)\n"
   ],
   "outputs": [],
   "execution_count": 11
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:19:17.974479Z",
     "start_time": "2025-01-26T10:19:01.823678Z"
    }
   },
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    val_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    tensorboard_callback=None,\n",
    "    save_ckpt_callback=None,\n",
    "    early_stop_callback=None,\n",
    "    eval_step=500,\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算\n",
    "                logits, deep_output = model(datas,return_deep_output=True)\n",
    "                deep_output= deep_output.mean(axis=1).reshape(-1, 1) # # 平均池化,尺寸为[batch size, 30]，求平均就变为[batch size],reshape成[batch size, 1]\n",
    "                logits=logits+deep_output  # 尺寸一致，相加，求损失\n",
    "                # 计算损失\n",
    "                loss = loss_fct(logits, labels)\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "                \n",
    "                # evaluating\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()\n",
    "                    val_loss = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss, \"step\": global_step\n",
    "                    })\n",
    "                    model.train()\n",
    "\n",
    "                    # 早停 Early Stop\n",
    "                    if early_stop_callback is not None:\n",
    "                        early_stop_callback(-val_loss)\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict\n",
    "                    \n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 10\n",
    "\n",
    "model = WideDeep()\n",
    "\n",
    "# 1. 定义损失函数 采用MSE损失\n",
    "loss_fct = nn.MSELoss()\n",
    "# 2. 定义优化器 采用SGD\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.0)\n",
    "\n",
    "# 3. early stop\n",
    "early_stop_callback = EarlyStopCallback(patience=10, min_delta=1e-3)\n",
    "\n",
    "model = model.to(device)\n",
    "record = training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    val_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=len(train_loader)\n",
    "    )"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/14520 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "19d442005f6846f69e07017df60ea7a0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:20:25.139963Z",
     "start_time": "2025-01-26T10:20:25.050037Z"
    }
   },
   "source": [
    "#画线要注意的是损失是不一定在零到1之间的\n",
    "def plot_learning_curves(record_dict, sample_step=500):\n",
    "    # build DataFrame\n",
    "    train_df = pd.DataFrame(record_dict[\"train\"]).set_index(\"step\").iloc[::sample_step]\n",
    "    val_df = pd.DataFrame(record_dict[\"val\"]).set_index(\"step\")\n",
    "\n",
    "    # plot\n",
    "    for idx, item in enumerate(train_df.columns):\n",
    "        plt.plot(train_df.index, train_df[item], label=f\"train_{item}\")\n",
    "        plt.plot(val_df.index, val_df[item], label=f\"val_{item}\")\n",
    "        plt.grid()\n",
    "        plt.legend()\n",
    "        # plt.xticks(range(0, train_df.index[-1], 10*sample_step), range(0, train_df.index[-1], 10*sample_step))\n",
    "        plt.xlabel(\"step\")\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "plot_learning_curves(record, sample_step=500)  #横坐标是 steps"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGwCAYAAAAJ/wd3AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAVSdJREFUeJzt3QeYU2X6NvA7ydRMoQ2doUjvXQQbShMsiAUVVlHXjn9R3LWsZcHeRV0Vy2dbRaygK0VHkCYd6Si9w9CZ3nO+63mTEzJDZiYzJCfl3L/rOiSTHJLzZjI5d95q0TRNAxEREZFBrEY9EREREZFg+CAiIiJDMXwQERGRoRg+iIiIyFAMH0RERGQohg8iIiIyFMMHERERGSoKIcbhcODAgQNISkqCxWIJ9uEQERGRD2TasKysLDRq1AhWqzW8wocEj9TU1GAfBhEREVXD3r170aRJk/AKH1LjoR98cnKyXx+7qKgIv/zyCwYPHozo6GiYBcvNcpuFWcvOcrPcoSAzM1NVHujn8bAKH3pTiwSPQIQPu92uHjeUfmGBxnKz3GZh1rKz3Cx3KPGlywQ7nBIREZGhGD6IiIjIUAwfREREZKiQ6/NBRESRqaSkRPVX8Bd5rKioKOTn56vHNouiIJY7Jiam0mG0vmD4ICKigM//kJ6ejpMnT/r9cRs0aKBGR5ppXigtiOWW4NGiRQsVQs4EwwcREQWUHjzq1aunRmn464Qpk1JmZ2cjMTHRL9/Gw4UjSOXWJwE9ePAgmjZteka/R4YPIiIKGGkW0INHnTp1/H4yLCwsRFxcnOnCR2GQyl23bl0VQIqLi89omK95fltERGQ4vY+H1HhQ+ItxNbecaV8Thg8iIgo4M/XJiGQWP/0eGT6IiIjIUAwfREREZCiGDyIiogBr3rw5Jk2a5JfHmjdvHmrVquX3octGMs1ol+ISBw5l5uNofrCPhIiIwkH//v3RrVs3v4SGFStWICEhwS/HFQlMU/Ox7Ug2znt5AV5fbwv2oRARUYRM9iVDTn0dosoRPyYMHwkxzkqeAkewj4SIyNzkpJ1bWOyXLa+wpEr7y3P74uabb8b8+fPxxhtvqBEesn3yySfqctasWejZsydiY2OxaNEibN++HcOHD0f9+vXVxF+9e/fGr7/+WmGzi8ViwYcffogRI0aoUNK6dWv8+OOP1X5Nv/vuO3Ts2FEdkzzXq6++Wur+d955Rz2HzA0ix3nNNde47/v222/RuXNnxMfHq7lYBg4ciJycHASSaZpdEmKdRS1yWFQTzBnMjUJERGcgr6gEHZ78OSjPvempIbC7voxWRELHli1b0KlTJzz11FPqto0bN6rLRx55BK+88grOOuss1fdCpjkfNmwYnn32WXXy/+yzz3D55Zdj8+bNaibQ8kycOBEvvfQSXn75Zbz11lsYPXo0du/ejdq1a1epTKtWrcLIkSMxYcIEXHfddVi8eDHuueceFSQkRK1cuRL33Xcf/vvf/6Jfv344fvw4Fi5cqP6vzFZ6ww03qOOQIJSVlaXu8zWkVZeJwoet1Bs/Pi6oh0NERCGsRo0aakItqZWQdVTEX3/9pS4ljAwaNMi9r4SFrl27un9++umnMW3aNFWTce+995b7HDfffLM68YvnnnsOb775JpYvX45LLrmkSsf62muvYcCAAXjiiSfUz23atMGmTZtUqJHn2LNnj+pvctlllyEpKQnNmjVD9+7d3eFDmo6uuuoqdbuQWpBAM034iLFZEWW1oNihIbugBLWTgn1ERETmFB9tUzUQ/phmPCszC0nJST5PMy7PfaZ69epV6mdZZ0VqHWbMmOE+mefl5amTfkW6dOnivi7hIDk5GYcPH67y8fz555+q2cfTueeeq5p5ZCZSCUoSLKSmRoKNbHpzj4QmCS4SOIYMGYLBgwerJhmp0Qkk0/T5kPY1vfYjt9A8Sy8TEYXi57E0ffhji4+xVWl/f8zQWXbUyj/+8Q9V0yG1F9JksWbNGnUyl/VXKhJdpv1fjk0Clb9Jbccff/yBL7/8Eg0bNsSTTz6pQocM1bXZbEhLS1P9WDp06KCaf9q2bYudO3cikEwTPoTezpdT4FvvZCIiMi9pdvFlDZPff/9dNW9IbYKEDmmm2bVrF4zSvn17dQxlj0maXyRciKioKNWRVPp2rFu3Th3f3Llz3aFHakqkD8rq1atVuSVMBZJpml2EPYY1H0RE5BsZNbJs2TJ1opZRLOXVSsgoku+//151MpUTufS9CEQNRnkefPBBNcJG+ppIh9MlS5bgP//5jxrhIn766Sfs2LEDF1xwgWpOmTlzpjo+qeGQ8s2ZM0c1t8jKw/LzkSNHVKAJJFPVfOjNLqz5ICKiykhzitQcSHOEzNNRXh8O6fApJ3UZSSIBRPpO9OjRw7Dj7NGjB77++mtMnTpVjc6RZhXpFCu1MaJmzZoqHF188cUqVEyePFk1wcjQXOlnsmDBAjVaR2pKHn/8cTVMd+jQoQE9ZlPVfCTozS6s+SAiokrIyVhqETzpJ/SyNSR6E4Zu7NixpX4u2wyjeRnK6ut06TLz6okTJ1Rw0F199dVq8+a8885TU7J7I2Fk9uzZMJq5aj7Y7EJERBR05uxwWshmFyIiCk133XWX6mPibZP7IoG5ml30obYFrPkgIqLQ9NRTT6n+Jt54NrWEM1OOdmHNBxERhap69eqprTxGjqQJFFM1u+jru7DDKRERUfCYK3zoNR8caktERBQ0puxwytEuREREwWOq8MFJxoiIiILPVOGD06sTEREFn6nCRyI7nBIRkUFk5lNZ1t4XFosF06dPh1lUOXzIHPAyd32jRo1Oe7GKiorw8MMPq1X9ZMlh2eemm27CgQMHEFJDbdnsQkREFD7hIycnB127dsXbb7992n25ubn4448/1Ip+cikL2WzevBlXXHEFQkECO5wSERGFX/iQle6eeeYZjBgx4rT7atSogbS0NIwcOVIt1XvOOeeoZX1XrVpV7mqARrLrHU4LS7wu6kNERAaQz9/CHP9sRblV29/Hz/73339f1d6XndBr+PDhuPXWW7F9+3Z1vX79+mrac1nS/tdff/XbS7R+/Xq1Cm18fDzq1KmDO+64A9nZ2e77Fy1apM6x0sogq9aee+652L17t7pv7dq1uOiii5CUlKRmRO3ZsydWrlwJU81wmpGRoZpn5MXxpqCgQG26zMxMdxOObP4Ua3W+6UocGnLyChAb7QwjkU5/Hf39eoY6lttc5TZz2UO53HJM8mVPTuLuE3lhDqwvNPHLt2fvZ5byOR7ZB8QkVLqfrBD7f//3f5gzZw4GDBigbjt+/LhaAfann35S56pLLrkETz/9NGJjY/Hf//5XdUn4888/0bRpU/fj6GX36dgcztdIWhiGDBmiwsWyZctw+PBhFT5kpdyPP/5YvaajR4/Gbbfdhi+++AKFhYVYvny5+7nkvm7duqkWCpvNhjVr1qhLf8yMKo8hzyPHII/pqSrvv4CGj/z8fNUH5IYbbih3Pvrnn38eEydOPO32X375BXa73a/H49BOFfmHmT8jMRqmIrVSZsRym49Zyx6K5Y6KikKDBg3Ut3Y5SSpFuVUODf6SmZUFRFfe9C4n1oEDB+Kzzz5TtRri888/V7UQUpNgtVrRokUL9/6yFst3332Hr7/+WgUF/UQt50H9S3Vl8vLy1L6ffvqpuv7WW2+pmg0JMy+88II6lz722GOIjo5W+0nNSN26ddX/1Vsj5HZpaZCgIjU3QoKMft+Zkt+hHJv0/ywuLj6t60XQw4ckIGl+kYT07rvvlrvfo48+ivHjx7t/lhcnNTUVgwcP9vsCOnJM0cvnoshhQd8L+iO1ln/DTaiScsuH0qBBg9Sb1ixYbnOV28xlD+Vyy8l37969qmkiLi7OeaOW5KyBOENyfsnKzkZSYqKqYfdFcrRdhpb4tK8MmLjzzjtVE4zUbkybNg3XX3+9qsmXMCVfnGfOnImDBw+qE7GclI8cOeI+d0lAkTL7ei6Lj49X++7atUvVXDRs2NB9n/xuJczIAI7zzz8fo0aNUrUzEpBku/baa937P/DAA7jvvvtUGJJam2uuuQYtW7aEv36fcpwXXHDBqd+nS1XCTVQgg4e0P82dO7fCF15+obKVJX9Agfgjkm4fRQ6goMQScn+kgRao1zTUsdzmY9ayh2K5S0pKVDCQE7FsbrakM35s1YxQ4IAlNrH0Y/uJ9OmQWoxZs2ap2o+FCxfi9ddfV8/10EMPqcD3yiuvoFWrVuqELCd5Of95Hotedl9YXa+RHqQ8/59+Xb9fmlTki7u0Ekhtiwz0kOORphoJRdL0MmPGDHXsEyZMwNSpU7321awq/fm9vdeq8t6zBip4bN26VXW+kSqqUBLrKnEuV7YlIqIKyDf7q666SvWr+PLLL9VAih49eqj7fv/9d9x8883qhC7TS0jTktRY+EP79u1Vp1Hp+6GT55MTvxyDrnv37qr1YPHixejUqROmTJnivq9NmzaqBkTCiZRB+oqEkiqHD6lqks4rsomdO3eq69LGJMFDkp/0qpVfliTe9PR0tbnb+oIsztU/JqeAw22JiKhieg3CRx99pK7rWrduraaTkPOfBAVpBvHXUvejR49WwWfMmDHYsGEDfvvtN9X59cYbb1Sja+S8K7UbS5YsUS0MEjDkC7+EFmn6uffeezFv3jx1n4SWFStWqPtCSZWbXSRYyBAend5fQ14kqdr58ccf1c/SXuVJXrz+/fsj2FzzjHGiMSIiqpR06qxdu7aas0oChu61115TQ2779euHlJQUNbjCHx06hQy2+PnnnzFu3DjV3CM/S/8OeU79fgkb0s/j2LFjqq+HdDCV/inS90Ruk/4qhw4dUscmNR/eBnaEVfiQAFHRHBmhPn9GnE2Oz8Ip1omIqFLS1OFtlm6ZOl36NHqSAOCpKs0wWplzpzTllH18ndR+yMgb6U9Ztj9JTEyMaiIKdaZa28WzzwdrPoiIiILDfOFDb3Zhh1MiIjKA9IGUocbeto4dO8KMAj7DaciGD9Z8EBGRAWR9sz59+ni9LzrEhkYbxXTh41SHU/b5ICKiwJM1VmQjEze7ODuccp4PIiIj+WsYKgWXvwaVRJm3wylrPoiIAk1GX+gjRmQdEvnZ16nQfQk0MoeUTPkdiBlOQ5UjSOWW4CHTx+sznJ4J84UPdjglIjKMvgCbrH/ibcjqmZ4MZVItmdrcX4EmHGhBLLc8X5MmTU5b0baqzBs+2OGUiMgQUtshK7PKBFgy87W/yKzasrqqLHJmpo6bRUEstzzfmQYPk4cPNrsQERmlvMXIzoScBCXQyFTkZgoftggot3kayVxirc7OMmx2ISIiCg7zhQ/WfBAREQWVicMHaz6IiIiCwbThI6+oBCWO0F4Ej4iIKBKZL3x4lFgCCBERERnLdOEj2gpYXcOi2fRCRERkPNOFD5mPJSHWOcKY4YOIiMh4pgsfwu5aXY4jXoiIiIxnyvCREOOq+eBcH0RERIYzZ/hwDXlhswsREZHxzBk+9GaXQja7EBERGc2U4cOuN7uw5oOIiMhwJg0fbHYhIiIKFlOGD32obS6bXYiIiAxnzvDBmg8iIqKgMfdoFw61JSIiMpzJO5yy2YWIiMhopgwfnOeDiIgoeMxd88FmFyIiIsOZMnwkcm0XIiKioDFl+LCz2YWIiChoTN3swnk+iIiIjGfytV1Y80FERGQ0U89wymYXIiIi45m65qOoRENhsSPYh0NERGQqpl5YTrD2g4iIyFimDB9RNitio5xFZ78PIiIiY5kyfJTu98ERL0REREYycfjgiBciIqJgMG/40Of6YM0HERGRoUwbPvROp9nscEpERGQoq9n7fOSy2YWIiMhQ5g0f+sq2rPkgIiIK7fCxYMECXH755WjUqBEsFgumT59e6n5N0/Dkk0+iYcOGiI+Px8CBA7F161aE7GgXru9CREQU2uEjJycHXbt2xdtvv+31/pdeeglvvvkmJk+ejGXLliEhIQFDhgxBfn4+QnK0C2s+iIiIDOX8+l8FQ4cOVZs3UusxadIkPP744xg+fLi67bPPPkP9+vVVDcn1119/2v8pKChQmy4zM1NdFhUVqc2f9MeTy3jXJGNZeYV+f55Q41luM2G5zVVuM5ed5Wa5Q0FVjseiSWKoJml2mTZtGq688kr1844dO9CyZUusXr0a3bp1c+934YUXqp/feOON0x5jwoQJmDhx4mm3T5kyBXa7HYHyyz4LZuy14Zx6DtzQkuu7EBERnYnc3FyMGjUKGRkZSE5O9m/NR0XS09PVpdR0eJKf9fvKevTRRzF+/PhSNR+pqakYPHhwpQdfnVSWlpaGQYMG4cjKA5ixdzPq1GuEYcO6IJJ5ljs6OhpmwXKbq9xmLjvLzXKHAr3lwhd+DR/VERsbq7ay5AUN1Isqj5sc73zOvGJHSP3yAimQr2koY7nNx6xlZ7nNJTrEyl2VY/HrUNsGDRqoy0OHDpW6XX7W7wsVdleHU04yRkREZCy/ho8WLVqokDFnzpxS1TAy6qVv374IJZxkjIiIKDiq3OySnZ2Nbdu2uX/euXMn1qxZg9q1a6Np06a4//778cwzz6B169YqjDzxxBNqThC9U2qoSHBPMsZ5PoiIiEI6fKxcuRIXXXSR+2e9s+iYMWPwySef4KGHHlJzgdxxxx04efIkzjvvPMyePRtxcXEIJZzng4iIKEzCR//+/dV8HhUNv33qqafUFsoSOL06ERFRUJh3bRe9z0dRCRyOak91QkRERFVk4vDhbHaRSpy8Ivb7ICIiMoppw0d8tA0Wi/N6Dke8EBERGca04UP6piS4+n3kcsQLERGRYUwbPoQ9hhONERERGc3U4SPRPdEYaz6IiIiMYurwoU+xzuG2RERExjF1+EjQ5/pgh1MiIiLDmDt8uJpdWPNBRERkHIYPru9CRERkKHOHD9doF9Z8EBERGcfU4cPu7vPBmg8iIiKjmDp8JLpGu+SywykREZFhTB0+7K4+H5xkjIiIyDimDh/ulW3Z4ZSIiMgw5g4feodTNrsQEREZxtzhg/N8EBERGc7c4UMf7cJmFyIiIsOYO3zoa7uw2YWIiMgwJg8fbHYhIiIymqnDh93d4ZTNLkREREYxdfhIdNV8FBY7UFTiCPbhEBERmYKpw4c+vbrgXB9ERETGMHX4iImyIsbmfAnY6ZSIiMgYpg4fwq6PeGGnUyIiIkOYPnwkcGVbIiIiQzF8sOaDiIjIUAwfnOuDiIjIUAwfrmaXXDa7EBERGcL04UOfaCybNR9ERESGMH340Ccay+VQWyIiIkOYPnzoQ22zOckYERGRIUwfPvQOp7lsdiEiIjIEw4d7ng+GDyIiIiMwfLiH2rLZhYiIyAgMH67RLpzng4iIyBgMH3rNB5tdiIiIDMHw4RrtwknGiIiIjGH68GF3dTjlJGNERETGMH34cE8yxg6nREREhjB9+NCnV2eHUyIiojANHyUlJXjiiSfQokULxMfHo2XLlnj66aehaRpCueZDOpyG6jESERFFEueZ149efPFFvPvuu/j000/RsWNHrFy5Erfccgtq1KiB++67D6HG7gofDg3IL3Ig3lUTQkRERGESPhYvXozhw4fj0ksvVT83b94cX375JZYvX45QZI8+FTak9oPhg4iIKMzCR79+/fD+++9jy5YtaNOmDdauXYtFixbhtdde87p/QUGB2nSZmZnqsqioSG3+pD9e2ceVfh8y1PZkTj5qxEZeN5jyyh3pWG5zldvMZWe5We5QUJXjsWh+7ujgcDjwr3/9Cy+99BJsNpvqA/Lss8/i0Ucf9br/hAkTMHHixNNunzJlCux2O4zwxEobMosseKhLMRonGPKUREREESU3NxejRo1CRkYGkpOTjQ0fU6dOxT//+U+8/PLLqs/HmjVrcP/996uajzFjxvhU85GamoqjR49WevDVSWVpaWkYNGgQoqOj3bcPfH0Rdh/PxdTbeqNns1qINOWVO9Kx3OYqt5nLznKz3KFAzt8pKSk+hQ+/N7tI8HjkkUdw/fXXq587d+6M3bt34/nnn/caPmJjY9VWlryggXpRyz62PsV6fonzvkgVyNc0lLHc5mPWsrPc5hIdYuWuyrFYA1HtYrWWflhpfpHmmJCfaIxTrBMREQWc32s+Lr/8ctXHo2nTpqrZZfXq1arJ5dZbb0WosrvWd+EU60RERGEYPt566y01ydg999yDw4cPo1GjRrjzzjvx5JNPIlTpzS65DB9EREThFz6SkpIwadIktYWLBH2KdTa7EBERBVzkTWpxBjUfXN+FiIgo8Bg+VM0HwwcREZFRGD48az7Y7EJERBRwDB8qfDj7fOQWsuaDiIgo0Bg+1NouzpqP7ALWfBAREQUaw4eaZMxV88E+H0RERAHH8FGq5oPhg4iIKNAYPjwnGWOHUyIiooBj+PDocMqhtkRERIHH8OE5zwdHuxAREQUcw4dHs0t+kQMlDi3Yh0NERBTRGD48ml0Eaz+IiIgCi+EDQIzNiiirRV3P5VwfREREAcXwAcBiscDuWtmWw22JiIgCi+HDJdE93Jbhg4iIKJAYPlzsrvDBmg8iIqLAYvgoO9EY+3wQEREFFMOHS4KrzwdHuxAREQUWw0eZmo8c1nwQEREFFMNHmZoPdjglIiIKLIaPMjUf7HBKREQUWAwfLlzZloiIyBgMHy6cZIyIiMgYDB9lJxlj+CAiIgoohg8Xe4ze54PNLkRERIHE8FFmZVuOdiEiIgoshg+XBFfNRw6bXYiIiAKK4aPsJGMc7UJERBRQDB9lm11Y80FERBRQDB8unGSMiIjIGAwfLgkxpyYZ0zQt2IdDREQUsRg+XOyuZpdih4aCYkewD4eIiChiMXy4JLhqPgSnWCciIgochg8Xm9WCuGjny8HhtkRERIHD8OFlivUcTjRGREQUMAwfXqZYZ80HERFR4DB8eJtojOu7EBERBQzDh4eEGK7vQkREFGgMH14nGmPNBxERUaAwfHjgyrZERESBx/DhpcMpp1gnIiIKHIYPL0Ntc9nsQkREFF7hY//+/fjb3/6GOnXqID4+Hp07d8bKlSsR6uyuDqes+SAiIgqcU3OK+8mJEydw7rnn4qKLLsKsWbNQt25dbN26FbVq1UK4dDhlnw8iIqIwCh8vvvgiUlNT8fHHH7tva9GiRbn7FxQUqE2XmZmpLouKitTmT/rjlfe4cVEWdZmV5//nDqbKyh2pWG5zldvMZWe5We5QUJXjsWh+Xj++Q4cOGDJkCPbt24f58+ejcePGuOeee3D77bd73X/ChAmYOHHiabdPmTIFdrsdRlp22IIp221oX9OBu9pzZVsiIiJf5ebmYtSoUcjIyEBycrKx4SMuLk5djh8/Htdeey1WrFiBcePGYfLkyRgzZoxPNR9Sc3L06NFKD746qSwtLQ2DBg1CdHT0affP2pCO+75ah17NauLL285GpKis3JGK5TZXuc1cdpab5Q4Fcv5OSUnxKXz4vdnF4XCgV69eeO6559TP3bt3x4YNG8oNH7GxsWorS17QQL2o5T12jQRncMotdITUL9RfAvmahjKW23zMWnaW21yiQ6zcVTkWv492adiwoWp68dS+fXvs2bMHoY7TqxMREQWe38OHjHTZvHlzqdu2bNmCZs2aIXwmGeM8H0RERIHi9/DxwAMPYOnSparZZdu2barj6Pvvv4+xY8cibCYZY80HERFR+ISP3r17Y9q0afjyyy/RqVMnPP3005g0aRJGjx6NUGd3r+1SAofDr/1wiYiIKFAdTsVll12mtnCj13yI3KKSUj8TERGRf5hnbRdNg2XfCjQ8uaLcXWKjrLA65xlDLqdYJyIiCgjzhI9tvyLq06HosvczwOG9Q6nFYnFPsc71XYiIiALDPOGjxYXQ4moirjgDlt2Lyt0twTXiRfp9EBERkf+ZJ3xExcDR/gp11brhu3J3S3B1OmXNBxERUWCYJ3xIt4+OV6tLy+b/AUX5XvfhyrZERESBZa7w0bQv8qJrwVKQBWz9xes+dtcsp5xojIiIKDBMFT5gsWJfrXOc19d/U/FEY2x2ISIiCghzhQ8A+2v1dV7Z8jOQn1HBFOsMH0RERIFguvCREd8MWkoboKQA+POnCvp8sNmFiIgoEEwXPmCxwOHqeOqt6UVf2TaHHU6JiIgCwnzhA4Cj41XOKzvnA1mHvNZ85LDZhYiIKCBMGT5QqwXQuBegOYCN33ud5yOXo12IiIgCwpzhQ3S+1mvTC6dXJyIiCizzho+OI9TQW+xfBRzb7r45gdOrExERBZR5w0dSfbXei+Ix3fqpScZY80FERBQI5g0fnk0v674GNK30JGMc7UJERBQQ5g4f7S8HbLHAsa1A+jp1k9092oXNLkRERIFg7vARlwy0vaRUx9NE12gXzvNBREQUGOYOH6VGvXwHOBzu6dU51JaIiCgwGD5aDQJiawBZB4Ddv7uH2haWOFBY7Aj20REREUUcho/oOKDD5c7r679xT68u2OmUiIjI/xg+PJteNv2AKK0IsVHOl4XDbYmIiPyP4UM0Px9IbADknwS2zeHKtkRERAHE8CGsNqDTqZVuOdEYERFR4DB86Dpf47zcPAt1YwrVVY54ISIi8j+GD12j7kDtlkBxHi7UVqibWPNBRETkfwwfOovF3fH0osL56pKjXYiIiPyP4cNL00un/FWogwzksMMpERGR3zF8eEppDTTsBhscGGZbhhw2uxAREfkdw0dZrqaX4bbFyGX4ICIi8juGj7I6XQUNFvSybkFU5t5gHw0REVHEYfgoK7kR9tXspa62OfJzsI+GiIgo4jB8eLGr4TB12eVEWrAPhYiIKOIwfHhxNHUwCrQoNCrcCRzaGOzDISIiiigMH15EJ9bGPEc35w/rvwn24RAREUUUhg8vZGG5H0r6OX9Y/y3gcAT7kIiIiCIGw4cXCTFRmOPogVzEARl7gb3Lgn1IREREEYPhw4uEWBsKEIPfLOc4b2DTCxERkd8wfHiREBOlLn9wuJpeNk4DSoqCekxERESRguGjnD4fYm5he2gJdYG848D234J9WERERBGB4aOcZhdRrNlQ3P5K541seiEiIgqP8PHCCy/AYrHg/vvvR7iIj7bBYnFez2rtCh9/zQAKc4J6XERERJEgoOFjxYoVeO+999ClSxeEEwlLCa5+H5m1uwE1mwFFOcDmWcE+NCIiorAXsPCRnZ2N0aNH44MPPkCtWrUQbuwxzqaXnKIS90q3as4PIiIiOiPOr/cBMHbsWFx66aUYOHAgnnnmmXL3KygoUJsuMzNTXRYVFanNn/TH8+VxE1zhIyOnAEXtRyB64SvQtqWhOOMQYK+NcFKVckcSlttc5TZz2VluljsUVOV4LJqmaf4+gKlTp+LZZ59VzS5xcXHo378/unXrhkmTJp2274QJEzBx4sTTbp8yZQrsdjuC5eV1NuzLseDOdiXoUEtD/78eR428PViTejN2p1wctOMiIiIKRbm5uRg1ahQyMjKQnJxsbM3H3r17MW7cOKSlpangUZlHH30U48ePL1XzkZqaisGDB1d68NVJZXJcgwYNQnR0dIX7fnFwBfblnECHLt0xrHMDWGttB+ZORBfLZnQc9grCSVXKHUlYbnOV28xlZ7lZ7lCgt1z4wu/hY9WqVTh8+DB69Ojhvq2kpAQLFizAf/7zH9XEYrM5mzREbGys2sqSFzRQL6ovj50Y57y/oMS5P7qOVOHDumcJrLmHgBpNEG4C+ZqGMpbbfMxadpbbXKJDrNxVORa/dzgdMGAA1q9fjzVr1ri3Xr16qc6nct0zeITDRGPZBcXOGyRsNDvXeX3Dd0E8MiIiovDm95qPpKQkdOrUqdRtCQkJqFOnzmm3hzK9w2luoSt8iM7XALt/d044du644B0cERFRGOMMp+Wwu+b5yJZ2F12HKwFrFJC+Hjj8V/AOjoiIKIwFbKitp3nz5iHcJMZ6qfmQIbatBgJbZgMbvgUufjx4B0hERBSmWPNRDrurz0eOZ82HcE849g3g/1HKREREEY/ho5IOpzl6h1Nd26FAdAJwYhewb2VwDo6IiCiMMXxU0uE0x7PZRcQkAO0udV7nSrdERERVxvBR1ZoPz6aXjd8DJV7uJyIionIxfJQjwTXaJbewTJ8P0fIiIL42kHME2Dnf8GMjIiIKZwwf5UhwjXZxTzLmyRYNdBzhvM6VbomIiKqE4aOSZhevNR+eTS9//g8oyjPwyIiIiMIbw0c57DEV1HyI1D5AjVSgMAvY8rOxB0dERBTGGD7Kkeiq+SgsdqCoxHH6DlYr0Olq53WOeiEiIvIZw0cl06v71PSy9Rcg74RBR0ZERBTeGD7KERNlRYzNWv5wW9GgE1CvA1BS6Oz7QURERJVi+KiA3dv6LmXJSreCTS9EREQ+YfioQIK3lW3L0vt97FwIZB405sCIiIjCGMOHD3N95JbX7CJqNXeOfIHmnPGUiIiIKsTw4cNcH+UOt/W20i0RERFViOGjAgkVTbHuqcOVgMUGHFgNHN1myLERERGFK4aPM5loTJdY17nei2DtBxERUYUYPnyYaKzC0S66ziNPhQ9NC/CRERERhS+GDx+G2uZUNNpF124YEBUPHN/ubH4hIiIirxg+fOhwWu4kY55ik4C2Q53XudItERFRuRg+KpDg6nCaU1mH07KjXjZ8Bzh8/D9EREQmw/Dhr5oP0WogEFcTyE4Hdi0K7MERERGFKYaPCiS4Rrv41OFURMUAHYY7r3PUCxERkVcMH/6YZMxb08umH4HiggAdGRERUfhi+PBlenVf+3yIZucCSY2Aggxga1rgDo6IiChMMXxUwK53OK1KzYfVCnR2LTa3/usAHRkREVH4YvjwYZIxn+b58Nb0snk2kJ8ZgCMjIiIKXwwfPkyvnuNrh1Ndgy5AShugpAD466fAHBwREVGYYvjwqeajGFpVpky3WLjSLRERUTkYPipgd4UPhwYUFDuq9p87ufp97JgHZB8OwNERERGFJ4aPCtijnc0uVR5uK+q0BBr3BDQHsHGa/w+OiIgoTDF8VMBqtbj7feRWtdOpYNMLERHRaRg+AjHRmK7jVYDFCuxbARzf6f+DIyIiCkMMH/6eYt1TUn2gxYXO61zploiISGH48HWisarMcuq16eVroCojZoiIiCIUw0cVhttWS/vLAFsscHQLkL7evwdHREQUhhg+KmF3re9S7fARVwNoM8R5nR1PiYiIGD587XBa7fDh2fSy4TvAUcX5QoiIiCIMw4ePHU6r3edDtB4MxCYDmfuBPUv8d3BERERhiOHDiJqP6Dig/RXO62x6ISIik2P4qESCa7RL7pnUfIgurqaXTdOB4sIzPzAiIqIw5ffw8fzzz6N3795ISkpCvXr1cOWVV2Lz5s0w5SRjnpqfDyTWB/JOANvn+OfgiIiIwpDfw8f8+fMxduxYLF26FGlpaSgqKsLgwYORk5ODcJQQewaTjHmy2k4tNsemFyIiMjHn13o/mj17dqmfP/nkE1UDsmrVKlxwwQUI20nGqrO2S1mdrwGWvgP8NRMoyAZiE8/8MYmIiMwePsrKyMhQl7Vr1/Z6f0FBgdp0mZmZ6lJqTGTzJ/3xqvK4ca6FbbPz/XA8dTsjqlYLWE7sRPGmH6F1cvUDCbDqlDsSsNzmKreZy85ys9yhoCrHY9G0wM357XA4cMUVV+DkyZNYtGiR130mTJiAiRMnnnb7lClTYLfbEWx/nbTg3T9taGzX8FDXM6/9aHvwe7RLn4705K5Y1vJBvxwjERFRsOXm5mLUqFGq0iE5OTl44ePuu+/GrFmzVPBo0qSJzzUfqampOHr0aKUHX51UJv1QBg0ahOjoaJ/+z+o9JzHyg+VIrRWPuePPP/ODOLoV0e/1hWaxoXjcRiAhBYFWnXJHApbbXOU2c9lZbpY7FMj5OyUlxafwEbBml3vvvRc//fQTFixYUG7wELGxsWorS17QQL2oVXns5ATnseUVlfjneBp2ABp2heXgWkRvnQH0vg1GCeRrGspYbvMxa9lZbnOJDrFyV+VY/D7aRSpSJHhMmzYNc+fORYsWLRDOEmL8NNTWU+eRzsv13/rvMYmIiMKE38OHDLP9/PPPVZ8NmesjPT1dbXl5eQjneT7yixwocfipharTVdLi5Zxq/eQe/zwmERGRWcPHu+++q9p7+vfvj4YNG7q3r776CuHI7lrbReSc6VwfuuRGQPPznNdZ+0FERCbj9z4fAey/GhSxUVZEWS0odmjILShBcpyf2tdkpdtdC53h4/zx/nlMIiKiMMC1XSphsVjctR9+q/kQHa4ArNHA4Y3AoY3+e1wiIqIQx/Dhg0R/rGxbVnwtoPVg53U2vRARkYkwfPjAHuvHKdbLTreuh48Ia64iIiIqD8NHFUa8+LXmQ7QdCsQkAhl7gL3L/fvYREREIYrhwwcJgejzIaLjgfaXO69zpVsiIjIJho8q1Xz4udnFs+ll4zSgJLQWCSIiIgoEho8q1Hzk+rvmQ7ToD9hTgNyjwI55/n98IiKiEMPwEcwOp8IW5ZrxlE0vRERkDgwfVRlqG4iaD33CMfHnT0BhbmCeg4iIKEQwfPjAPcmYv0e76Jr0Bmo2BYpygC2zAvMcREREIYLhI1iTjHmyWE7VfnDCMSIiinAMHz6wx+jNLgHo86HrPNJ5uTUNyD0euOchqsSx7AIcySoI9mEQUQRj+PBBQmyAm11EvXZA/c6Aowj488fAPQ9RORwODR8t2ol+L8zFgFfnYd8J9j8iosBg+PBBghE1H2WnWycy0P6TeRj94TI89dMmFBQ7kJlfjCd/2Bhxq1QTUWhg+Ajm9OpldbraeblrEZCxP7DPRQRZUkjDd6v24ZLXF2DJjmOIj7Zh3IDWiLZZMPevw5i5Pj3Yh0hEEYjhowrNLrmBDh81U4Gm/eSUAGz4LrDPZRL84l5x3467Pl+FB79Zi6yCYnRvWhMzx52PBwa1wd39W6l9JvxvIzLyOPMuEfkXw0eodDg9remFE46diYLiEvzj2/V4YpUNi7cfC/bhhJy0TYcwZNIC/LzxkKrl+OeQtvjmzr5okZKg7r+nf0uclZKgOp6+NPuvYB8uEXn46PddeHODDQdO5iFcMXxUcahtwNvAO1wJWKOA9HXAkc2Bfa4IlVdYgjs+W4Uf1h5EVpEFd09Zg1W7TwT7sEJCVn4RHvp2LW7/bCWOZheiTf1ETB97LsZe1ApRtlMfB3HRNjw7orO6/sWyPVi1myOwiELBhv0ZePHnLdieZcGD365HcYkD4Yjhwwd2V7NLsUNDYaB/0Ql1gJYDnNfZ8bTKsguKcfPHyzF/yxHERVvRLFFDbmEJbvl4OTYdyISZLdtxDEPfWIivV+5TU8vcccFZ+PHe89CxUQ2v+/dtWQfX9myirj/6/XoUFofnhxxRpChxaHhs2no4XN+BV+4+iXfmbUc4YvjwQYKr2SVg67uU1WXkqaYXdlrw2cncQjViY9nO46q26qObemJshxL0bFpTjd646aNl2HEkG2aTX1SCZ2dswvUfLMW+E3loUiseU28/B/8a1l7VcFRE9qmdEIMth7LxwcIdhh0zEZ1uyrLdWLsvQ32+XdHUeS56Y87WsKzZZfjwgc1qUd+iDRnxItoOBaLtwImdwP4/Av98EUD6Jlz//lKs3XsSNe3RmHJ7H/RuXgtSafX+37qjQ8Nk1czwtw+XqWGlZqqiveI/i/DBwp0qx17XKxWzxp2PPmfV8en/10qIweOXtlfX35yzFbuP5QT4iInIm8OZ+XhptrMp/sFBrXBxIw2Xd2mgakPu/2q1alINJwwfobK4nKeYBKDdpc7r7HhaqYMZebjuvSX4Kz0LKYmx+OqOvujSpKb7/uT4aPz372ejZd0EHMjIVwEk0mfwLNGAd+fvwIh3fle1FimJMfjgpl548ZouSIqLrtJjjejeGOe2qqPm/3hs2gbO/UEUBE/P+FONSuvapAZu6J2qmk4nXt5e1WTuPZ6n5uUJJwwfVR3xYkSzi9DXepEhtw6DnjMMyTfxaycvwY6jOWhUIw7f3NUXbRsknbZfncRYfH5bHzSuGY+dR3Nw4/9bhozc8Pqm4CvpAS894V/7dRuKSjQM6VgfP99/AQZ1qF+tx7NYLHj2ys6IjbJi0bajmL6Gc9AQGWnBliP439oDsFqgOoJLbbyQLxKTruumbp+2ej9+CKO/TYaPUJtoTNfyYiC+NpBzGNg535jnDDPbDmdh5HtLVD+G5nXs+PquU0NFvWlYIx5f3NYHdZNiVS3JzZ8sN+73aZCj2QW46eNV2JVtUbV1r17bFZP/1lOFrzPRPCUB9w1ora4//dOfOJFT6KcjJqLK+mw98cMGdX1Mv+bo1Lh0B/FezWu7/zYfn7YBe4+Hx7IIDB8+SohxTTRmRLOLsEUDHa90Xueol9NsPJCBke8txaHMAjVc9Os7+6JJLbtPJ9HP/95H9QtZveekGnIqf9yRMtLn1k9WYPfxXNSO1fC/sX1xdc8mqubCH24//yz1Wh/PKcRzM//0y2MSUcXe+W0bdh/LRYPkODw4uK3Xfe69qBV6NaulmmXGTV0dFsNvGT58ZHfXfBh4otKbXtZ8Abx0FvBOX+DTK4Dvbgd+fgz4/Q1g7VRg+1wgfQOQfURWB0Ok+2PPCdzw/lJ1EuzUOBlT7+iLeslxPv9/aZb59JazVaCUCcjunbIaRWHwx1oRGQZ7139XYd2+DNSyR+Pu9iWqLdifYqKseP4q59wf36zahyWcvI0ooLYdzsa7851Daf99eQd338OyZI6e16/rhqTYKPyx5yTemrsNoc57Seg0ia65PuTbpWFSzwFS+wB7lwG5x5xbZSw2ICEFSKwHJNYHEurBak/BWYePwrIxH6jR0Hm73B9XUxr0EU4Wbz+K2z5dqebukKT/0S29kVzFDpSia2pNfDimt5oT5Nc/D+Ef36zF6yO7wepqSw231WhlinTpj2GPseGDG3tg/7rfA/JcPZvVxug+TdXEYzLfgEzHXtlwXSKqOk3T8Pj09arf1kVt6+KSTg0q3D+1th3PjOiEcVPX4K25W3F+6xTVJBOqGD58lODqcDrxfxvx2ZJdaN8wWW0dXJf1k2P9Vr3tZrUCt/4M5Bx19v3IPgRky6XH9RyPnyWcaCWu+w5Je416GDk1qO+r+6eUfnxbjAonzqBSr1RgcV/Xb49JDHpQ+e2vw2otEhl1cV6rFLx/U093R+DqkEm03v1bD+dsqGsOqG8Vz1zZyf+/xwB/QMlKtNIZLcpqUf07pDf8/nWBe86HLmmHXzYdUp183523Xa0FQ0T+NW31fizdcVxN8/DUcN8+l4Z3a4z5m4/g+9X7VQiRLwc14qv+5cwIDB8+urxrI8zbckQN0dx+JEdtP6076L5fqrr1QOLcktC6XpKqqj4j8oZLrOvc6neseN+SImdQKRVMDqEkMx0Ht65Fo2QbrDlHnLfnZwAlhUDmPudWGZl3JKGuRyCpX05gqQdE+7e6X8xcf1C1Zcq3gIHt6+M/o7r75Rv3xe3qq+rK+6auVt/mE+Oi8Mgl7cImgEiV7CeLd6nrr47sigva1EVRUWBH8ciHmVQBS3OVhA/522hVLzGgz0lktgkTn53h7FclnUmlVsNXE4d3xMrdJ7DneC6emL4Bb1zfLSQ/zxg+fCQf6sv/NUCFj00HM/HnwSz8qS4z1TfAE7lFqv+A5yJm8k1UPpTdYaR+EprVtquOkWccSsrrpJrc0Ll5cBQVYVXxTNQfNgzWaFcKLsp3hZMjp2pKcjyuu28/DBTlAEW5wMndzq0ysclAfC1nCImKBaLinLUscqn/rLayt8UCttjSP0fFYeGuLHy+cC86a9Ho07Yxxg9JQHT23tP3tVYvjMjJU0a9PPL9erw3f4dqxpG1TkLd1yv2uicdeuKyDupbj1Eu7dwQ37Xdh982H8G/pq1XM6aGY5MVUSh6cfZmHMspROt6ibjtvLOq9H9l+K0EjmsmL8GPaw+gf9u6uKqHc5mEUMLwUQWSHqVjo2z929Zz3y6jJbYeylZBxBlMnJtM6S1DOmWbtvrU48hntAz7bFbHrramtRNcl3Y0rWOvVh+GKouOA2o2dW6VKcgu3bzj2fTjDiyu20oKgIJM5+Yn58sW4/pBss/kcna0RrvCiDOQRNlicFFeEWwHXy0dhNz7nApC10fFonX7HMzZcgInf7VheXoTnN2ynjM0SaiTx7ZFuS7l56jy73Pfpu+n/xzjbErz06q0j3zvbFu568KW+Pt5LWD034JUBQ9+fQGW7zyOb1btxXW9fXgvUUSRZr+NrjWTOjZKDslv2OFm1e7j+HL5HnVd5vSozhfV7k1r4f4BrfFq2hZV+9GzWS00q1P+NATBwPDhB1L937lJDbV5/lHKbJqymJm7huRIjqoKyysqUVN8y+ZtuXdpwmlaJ0HVkuihROavkFk7A1JjUpnYROdWu5IELjNfSnOOhJC8E84gUixbvuvS87rr8rR9nJe5ebnYfeg4cnNzEIMi1LdbUDdeg8W9TyFQnAdoHqNUHEVAoWzO9VvkYzBZrqTv860zpWx67pMKhUAsKmyxnh5MVCiJ8hJk9HBT+r5jeRqytp/EszYbmtWtgb5aPWB2lOuxo2DVLGh7cCesCzcCUa7/Lx2R5VJqh1z7qevu+1yb537unz33k/us6nqqNQpP9YvC2wv24dMZxzGocT/UTowvvV+px3cdI09QYU86Oaf9eQjvzd+uRlcI+ayS2rcruzXCWXXZDFcdRSXOWYTFyF5NcHaL6ncYveeiVli49SiW7zqu+n/IBIzRHitXBxvDR4DINwCZTVM2z5klJZQcyS7AnmO5auy2zMmw51iO6zJXVbVJE86J3JNqnRJPSXFRGNS+PoZ1bojzWqeE3igDOanE13Ru1ZSRV4S35mzFp+t3qf4dMpPffRe3xn0DWnn/VlVSXCbInAoxxfk5WLZ4Ifr07IoordjrPqdCkfNnraQI6/Ycxe4jGYhGCbo2sqNhUhQs0p/GUezsVyMhRy7d14u93Oa6LEvCkhyDbNUkq7JcJZ8hssl6UstL3y/vinZyJR0BJ4PBr9XnL3vfx/+kAohHuFFhyOpx3cvtemhyX7d4vV3K3u/YCdi+/Lh0eFL76I9d9vktZZ6zottdj6Oe33qGWyWPYa3CY5Q4kJy7Bzi8CYiOOX0fieJln7Psbepnb7ed+rmgRFOds99buAvbj8pkVhb1hchmsajPM1n/R7YuTWrgym6NcVnXhqiX5PsweLP7+PedqqZcvoA+MtS5plJ1yWfn69d3w9BJC7Bm70n1eylvnpBgYPgIRtNNUpzavA2DkqG8EkL2HM/xCCe56g0ps1dKL2bZZGTGgPb1MLRTQ9WmF4ggIt9udh3LwdbD2WpWPQlSgSKT4khV42tpW1T4EjK87LFL26NVvdOnS3eTWgKbq2amDK2oCEeTjkFrNQjQ+7pUQj6mu2gavvhunVp6Xpp55IN0/KA2uLBN3apVK0tNkEyNX14wcYeWwtIBRr9PbvfY70RWDj6cvwX5+floVjMaN/RqpAKSGuEkwUg9VwlKSoqwe+cONEttDJtFPwbXPmpf11bq/8l9jjL7yX2O038u8/9KSkpQUFgIGxyItmqwStCr8HWRx5DH9X/HWDld1pUr2ZtgJvLuvggBqq3zIDlT1txW6267MoXmCihalAUlmmyAdsQCR5oFWpoFuVYroqKi1Ldui9dg4y0A6YGpzO2eYQgWdQK7MCsLUQdedjVplr6/4v+vB2Hfnuu04yp7f9+xQOMe1X5t953IxetpW9X1R12rSZ8p+cx+7qrOqnP4f37bpkYJ+rqoZKAxfIQYCRUdGiWrrWwQWLXnhBr1MWt9OtIz89U3ENlkboeL29VTNSIXta2HeNdsrFV1KDNf1bas3Se1LhnqMivfeSKRv6/zW9dVVYFSkxMbZfPrugVP/7RJhRwhnXRlJVXPfjVGkoDx/FVd0KBGPD5cuENN3HXzxyvUvCLyzUGG6Pr4QK5wFHXGI4BkQrVrJi/GjpwmqhOaqkK1e/9wkg7G62fOROqwYbD5GLrOhLwTXv7fRnz8+y6k1o7HL/dfiPgoS5nQImHHFWQ8g447iOjXS8pcd1Tp9uKiQqxZ/Qe6de0COYTT9/F8vpJyjqO82x1eNu3Uc1R4f5nNUYKC4mIcOJ6Do1n58luDFZraLB7X9Z8l2Dmvy20OWC2aKl+URVMBUzaLoxjxMVGQBbit7uORRQD1Y3Bdevv5DMgxSfnl5ZZTd3TZfC4PLzkzAIOw5KlUPWsoLFTdccQZhY8JP25STfJnN6+Na3v6r4PoZV0aqeG3MjHgA1+twaxxF6CGPfjDbxk+woSMJOjdvLbanri0A1bvPYlZEkQ2pKu+IzLsV7b4aBsualdX1YhIINHXpCkrM78I6/dlqOq4da6wIYGmLFlMTPqcSDCQkCCbTE0uVaoje6WeFpKqOnufTNM996/D6mepapQ5I0ad3VTN2BdMUmUptR1j+jbD5Pnb8dmS3Wr42g0fLFUrvEoI6dG0liHHIlP6y7Tp0mdIFs/77O9no2Y5wSNY5PWYvSFdra755tytePiSdoDV+GOU2q79u+PQueNQ5GoWFZ4z84rUpSw5Lu97db3AeXum63a1X36R+vuREH95l0YB+4CWpdHfmbcdU5btQaFrZt1+Levg1nNbqLwqQfNEbqGzCTanUP3suckxl8t1Epa1jno0q6U6Gsomw/71xci8cock5+WOo1n4eNEO/LB6P4pLJFhoaJkSj1v6NVMjnWLUn2d5oebUbfJtfs6mdMz5Mx37T+Sop5IQVSPOhv5tUnBx2xS0b5DoCjBamcfSg5O325zPUVxcjOXLl+Hs3r0QZbOdHrYq+f/Osvu6r+f9OP3++h2q/Z74ZWO6muxQRkg+O8L/cw1NuKIjVuw6jl3HcvGv6evxnxu6B71zsEULsfWxMzMzUaNGDWRkZCA5ufonNm9k/oOZM2di2LBhiDbgG6ER5Ncn38ylRmTmhoPqw98zOEiTzOD29bDvr9VIatYJGw5kYc2+k+pEVpZ8NrWpn4SuTWqqGUCluUGmIpfqUlk99ttV+9R2MONUSOncuIaqDbmiW2OfJ7ORMeyTft2Kz5fuRrFDU39wsmCS9O3w5we+P3/fUiv09m/bVNOQ9EUREu4koJRd6MnfHdBk/Zl5m52h79u7+lbcDBXE97mMwJFjld/nPf1bqjlTpIZMJklyX0bb1PsyroJL+USS5kc9FOjBwXmb63b39WJkl9nvRHYe8h0W97mlOqQfw+AO9XFNzyaqxq/CE7ePZJi+BFl538tEeUI6FMp76JwqVIXLe0LCyYmcIhzLKVCXhzJykbZiE45oSdjm5W9bpt3u1rSmCswSRuS6t1F1q/ecUMPNf96U7n79ZH8ZUTWgXb1qD6fWP6dkReT/rT2ompB1sgyAvMZ9WtRG7xa1q9S8Gwmf6TkFxRj02nw1QEH+bmQSv0CUW2q1r353sfrMffmaLri2VyqCef5m+IjAYW8zVNPMQZVyKyJV5DKCppsrbMg6KZXNGFri0LBw6xF8vXKvOtnoJ2I5ccj0v9f1SlUfpN4+pORDUz54JXhIx1IhE4b9a1i7gPSOD8TvW77JvTVnG779Y596LcTQTg1UjY0Et0BMmy4zHcqJ+4vbzlEnglB+n9/535X4eaPMrhsaYmxWJMdHqbkPpMN2suvy1HWPn+OjVf8qCdibD2W5H0NmL76ye2Nc06OJmqunqo5lF+D9BTvw6ZJdyC9yhg75PUrokBoPf3wD9fydS5epP/aewB+7T6h1kGQBRVmOwJM8Zdv6Sap2RAKJNPdKZ8dlO4+79xnYvp4KHf6eolv6d8kov+mr9+PnjenIKXNsEj56N6+lgog0QUgzbHmvUSR8pj87YxM+WLjzVJOlD83m1S33O/O2qbmBpKl+xn3nV7gKeHUwfJQjEt6ovpJfq0yENmvDQVUrcvhkNnq0qKvGf0vNhtRqnOky61IFLCdGmezK88Na/oiu7ZmqVlSVDxI5Fvnm/vSMTe4aF/ngk4mxZNROoATy973jSDbemLNVTeIjf0Hy2Ti8ayPcP7CNWjnXH6RJSk5a8q37w5t64aJ29UL+fS7NBO8t2IGMvEJ1oi0oLqnwUubIkVoAWRivLGkC0YOBZ0hIio1WtSqlbo91Xo+PAv5Y9jsuGzIAtRLjq9URWw/xEkJ+WLPf3QFaSEiX2pArfGiWkdfi/YU78OniXe6Tf7fUmip0yLob/qz2ruh3Lid7+ft0hpGTWOWa/dIbqbWSoHXHBWf5PUx7k1dYor7MSJPA8l0nsGF/hjvU66Q5VgKQBBEJJDKfiD5kNNw/0zcdyMTl/1mkyvzxzb0D/jcuzzP6w6Vq2nZZhuHbu/v5dfgtw0c5wv2NGqrl1qtUpTbkxzUHVJW4ZydVuV/Gm4s6CTEYP7iNqiEJdL8OI37fm9Oz8HraFsze6BzXKkFBviH/34BWaibbisjrIidgZ1+E4lJNDVL9Ld+GxKvXdlVBLpLf51LLI/0fJIwI+SZenfeHv8suoUj6JEkQ+W3zYfeJUWpUBrmbZVJKHWtGbhE+XLRDdcDVF6KU5kkJHdIMGoi29qqW+3BWPv7YfVK9zySMSFOq1OD9/fwWagLEYDZBSE2NzE2xYudxrN57wl1bpJNv7d2b1sTZzeugR2oyDm5cihGXG/Nel6YzmVRv2c5jWLbjuOqX07R2vPrC0bxOgrpsIXM0pVQ+WaTDoeHqyYtVeYd1boB3RstMQ4F/nx/MyMMlkxaqL34vXt2l3JVyA33+ZodTOmPyYSrfCGV7/NIOmL3xIL5asVela+mgKqJtFtWhbuzFrYyZwdUg0idm8o09Vefd19I2q+nGv1q5F9+v3ocrujZGQqztVD8FvV9CgfN6dn6xan+tyKND21UpeIQraaaLs9pCbu6aGFdzomxy4pGaEAkiMvRdmjdlq5sUixHdG6uOmBJQ/t/Cne4ALksrSOiQJoxgd/DzJEP99XKFEukgLydFvUZUwt+GAxkqiMhJX2pIJKj/vu2Y2oQFNry383fV90pqRTo2qqEW/Kzlh6Gq0tdr6Y5jqjlq2Y5jak2vsqT/ij7Rmif5oiVhRCZfk0ByKqDYVS3dlyv2qOAhJ/8nL6tk3S4/knApC85J5/VgvicZPsivpL1yRPcmapNOqt+t2ofsghKM6dcs5Kb39SeZ3fbjW85WUyO/+ssW1ab93R++zawq3WPkA6hU00JctJrHRUb+UGiQkHHb+WepbeOBDFezzAEVSqR5TDadNCs+MKg1BndowDVvzjD8SZ8U2e68sKWqLdhyOMsZRnadwPKdx3Ao89Rin/L70MnJtUMjPZAko2PjGpWecGXkoIQMqdWQ2o2y/eb0vjLSr006yErtpjRhyXxIO4/K3ExymasCidSKyCY1S2WlJMa4a8UeHNwGDWoYOxFbIOdsCnr4ePvtt/Hyyy8jPT0dXbt2xVtvvYWzzz47UE9HIUjCxvgQmlHPCD2b1caU28/B4u1H8dtfh9U3eQkTibGlg0Wy3Oa6nhBjC6lvxVQ5+XYt26ND26vaDtUs89dh1YFP+v1IEwZDh//Ja9quQbLabuzbXDU/fDl9Jhp26I3Nh3JUXx3ZJBDI6BHZZAirTkaMSRCRmhH5/Z1VN0E1naqajZ3HSo0WVM9ngZpOoE8LZ9iQ0Ullh7l7Lquhk5pOmSRSQsmuozkqxDgvc3A0u1Bt6v82roEbz2kGMwpI+Pjqq68wfvx4TJ48GX369MGkSZMwZMgQbN68GfXqBWfiKCIj9WuZojaK/G/mQzo2UJt8K2fgMF6NGKB/m7oY1LGR+zaZs0U6c8rmDCQZal6hk7lFpZpsypI+W50kbJxVB+ecVVt1dK1OM7F8qZBmIG/D8PVgsv9knpq3KdhzGkVU+Hjttddw++2345ZbblE/SwiZMWMGPvroIzzyyCOBeEoioqBi8AgdEhikacRz/hR99XEJIhJIZAVyGbUmtVUSNqRmQ8KGPztgVjWYmInfX+XCwkKsWrUKjz76qPs2q9WKgQMHYsmSJaftX1BQoDbP3rJCqtNk8yf98fz9uKGO5Wa5zcKsZWe5Ky+3WnCxvl1tV3dvWM5eWli8hkUh+vuuyvH4fajtgQMH0LhxYyxevBh9+/Z13/7QQw9h/vz5WLZsWan9J0yYgIkTJ572OFOmTIHdXvFQRSIiIgoNubm5GDVqVHgMtZUaEukf4lnzkZqaisGDBwdkno+0tDQMGjQobOY/8AeWm+U2C7OWneVmuUOB3nLhC7+Hj5SUFNhsNhw6VHqKZfm5QYPTx5THxsaqrSx5QQP1ogbysUMZy20uZi23mcvOcptLdIiVuyrH4vdutjExMejZsyfmzJnjvs3hcKifPZthiIiIyJwC0uwizShjxoxBr1691NweMtQ2JyfHPfqFiIiIzCsg4eO6667DkSNH8OSTT6pJxrp164bZs2ejfv36gXg6IiIiCiMB63B67733qo2IiIjIkzmnViMiIqKgYfggIiIiQzF8EBERkaEYPoiIiMhQDB9ERERkKIYPIiIiMhTDBxERERkq6AvLlaUvsluVBWqqshiPrLonjx1K8+EHGsvNcpuFWcvOcrPcoUA/b+vn8bAKH1lZWepSVrYlIiKi8CLn8Ro1alS4j0XzJaIYSBahO3DgAJKSkmCxWPyeyiTU7N27F8nJyTALlpvlNguzlp3lZrlDgcQJCR6NGjWC1WoNr5oPOeAmTZoE9DnklxVKvzCjsNzmYtZym7nsLLe5JIdguSur8dCxwykREREZiuGDiIiIDGWq8BEbG4t///vf6tJMWG6W2yzMWnaWm+UONyHX4ZSIiIgim6lqPoiIiCj4GD6IiIjIUAwfREREZCiGDyIiIjKUacLH22+/jebNmyMuLg59+vTB8uXLEU6ef/559O7dW838Wq9ePVx55ZXYvHlzqX3y8/MxduxY1KlTB4mJibj66qtx6NChUvvs2bMHl156Kex2u3qcf/7znyguLi61z7x589CjRw/Vk7pVq1b45JNPEApeeOEFNevt/fffb4oy79+/H3/7299U2eLj49G5c2esXLnSfb/0FX/yySfRsGFDdf/AgQOxdevWUo9x/PhxjB49Wk1EVLNmTfz9739HdnZ2qX3WrVuH888/X/1tyKyJL730EoKlpKQETzzxBFq0aKHK1LJlSzz99NOl1oqIhHIvWLAAl19+uZoJUt7T06dPL3W/kWX85ptv0K5dO7WPvMdmzpyJYJVd1ix5+OGH1XEkJCSofW666SY163W4l72y37mnu+66S+0zadKksC93uTQTmDp1qhYTE6N99NFH2saNG7Xbb79dq1mzpnbo0CEtXAwZMkT7+OOPtQ0bNmhr1qzRhg0bpjVt2lTLzs5273PXXXdpqamp2pw5c7SVK1dq55xzjtavXz/3/cXFxVqnTp20gQMHaqtXr9ZmzpyppaSkaI8++qh7nx07dmh2u10bP368tmnTJu2tt97SbDabNnv2bC2Yli9frjVv3lzr0qWLNm7cuIgv8/Hjx7VmzZppN998s7Zs2TJ1jD///LO2bds29z4vvPCCVqNGDW369Ona2rVrtSuuuEJr0aKFlpeX597nkksu0bp27aotXbpUW7hwodaqVSvthhtucN+fkZGh1a9fXxs9erR6b3355ZdafHy89t5772nB8Oyzz2p16tTRfvrpJ23nzp3aN998oyUmJmpvvPFGRJVb3oePPfaY9v3330uq0qZNm1bqfqPK+Pvvv6v3+ksvvaTe+48//rgWHR2trV+/PihlP3nypPpb/eqrr7S//vpLW7JkiXb22WdrPXv2LPUY4Vj2yn7nOrlfytaoUSPt9ddf18K93OUxRfiQN+/YsWPdP5eUlKhf7PPPP6+Fq8OHD6s38Pz5891/tPIGkg9r3Z9//qn2kT9g/c1vtVq19PR09z7vvvuulpycrBUUFKifH3roIa1jx46lnuu6665T4SdYsrKytNatW2tpaWnahRde6A4fkVzmhx9+WDvvvPPKvd/hcGgNGjTQXn75Zfdt8nrExsaqDxwhHyzyWqxYscK9z6xZszSLxaLt379f/fzOO+9otWrVcr8W+nO3bdtWC4ZLL71Uu/XWW0vddtVVV6kP00gtd9kTkZFlHDlypHrNPfXp00e78847NSNUdBL2/OIh++3evTtiyo5yyr1v3z6tcePGKjjIlw/P8BEJ5fYU8c0uhYWFWLVqlaq29Fw/Rn5esmQJwlVGRoa6rF27trqUMkqVpWc5pVqtadOm7nLKpVSx1a9f373PkCFD1CJFGzdudO/j+Rj6PsF8raRZRZpNyh5XJJf5xx9/RK9evXDttdeqpqLu3bvjgw8+cN+/c+dOpKenlzpuWVNBmhQ9yy5Vs/I4Otlf3v/Lli1z73PBBRcgJiamVNmlSe/EiRMwWr9+/TBnzhxs2bJF/bx27VosWrQIQ4cOjehyezKyjKH43vf2WSdNEFLeSC67w+HAjTfeqJqFO3bseNr9kVbuiA8fR48eVe3InicfIT/LH3g4kjep9Hs499xz0alTJ3WblEXecPofqLdyyqW310G/r6J95GSdl5cHo02dOhV//PGH6vNSVqSWWezYsQPvvvsuWrdujZ9//hl333037rvvPnz66aeljr2i97VcSnDxFBUVpQJrVV4fIz3yyCO4/vrrVYiMjo5WoUve69LOHcnl9mRkGcvbJ9ivgWefLukDcsMNN7gXUIvUsr/44ouqHPJ37k2klTvkVrUl32oCNmzYoL4RRjJZLnrcuHFIS0tTHaPMRAKmfMN57rnn1M9yEpbf+eTJkzFmzBhEqq+//hpffPEFpkyZor79rVmzRoUP6aQXyeWm00mt5siRI1XnWwnikWzVqlV444031BctqeUxg4iv+UhJSYHNZjttBIT83KBBA4Sbe++9Fz/99BN+++03NGnSxH27lEWamE6ePFluOeXS2+ug31fRPvKtQ3rdG/0HefjwYTUKRRK+bPPnz8ebb76prktaj7Qy62SUQ4cOHUrd1r59ezVyx/PYK3pfy6W8fp5klI/0mK/K62MkqXLWaz+kuUyqoR944AF3zVekltuTkWUsb59gvwZ68Ni9e7f68uG5bHwkln3hwoWqTNJkrH/WSdkffPBBNUozEssd8eFDquV79uyp2pE9v1XKz3379kW4kPQvwWPatGmYO3euGoroScoo1dSe5ZR2PjlZ6eWUy/Xr15d6A+t/2PqJTvbxfAx9n2C8VgMGDFDHK99+9U1qA6QKXr8eaWXWSZNa2aHU0g+iWbNm6rr8/uXDwvO4pZlI2n49yy7BTEKcTt478v6X/gP6PjIEUD7sPcvetm1b1KpVC0bLzc1Vbdie5MuDHHMkl9uTkWUMxfe+HjxkaPGvv/6qhpp7isSy33jjjWqIrOdnndT2SRiXZteILLdmkqG20lP8k08+UT2G77jjDjXU1nMERKi7++671dC7efPmaQcPHnRvubm5pYadyvDbuXPnqmGnffv2VVvZYaeDBw9Ww3VlKGndunW9Djv95z//qUaOvP3220EfdurJc7RLJJdZevhHRUWpoadbt27VvvjiC3WMn3/+eanhmPI+/uGHH7R169Zpw4cP9zocs3v37mq47qJFi9SoIc+heTKKQobm3XjjjaqHvfytyPMEa6jtmDFjVG9/faitDDuUodEyIimSyi0juGTot2zyMfzaa6+p6/qIDqPKKMMu5X32yiuvqPf+v//974APu6yo7IWFhWpYcZMmTdTfq+dnnecIjnAse2W/87LKjnYJ13KXxxThQ8jcDXKSkvk+ZOitjJMOJ/Jm9bbJ3B86+WC655571FArecONGDFC/dF62rVrlzZ06FA19ls+1B988EGtqKio1D6//fab1q1bN/VanXXWWaWeI9TCRySX+X//+58KThKc27Vrp73//vul7pchmU888YT6sJF9BgwYoG3evLnUPseOHVMfTjJXhgwvvuWWW9SHoCeZR0KG9cpjyIlfTnzBkpmZqX6/8rcaFxenfhcyN4LniScSyi3vN29/zxK+jC7j119/rbVp00a992XI+YwZM4JWdgmc5X3Wyf8L57JX9jv3JXyEY7nLY5F/jK1rISIiIjOL+D4fREREFFoYPoiIiMhQDB9ERERkKIYPIiIiMhTDBxERERmK4YOIiIgMxfBBREREhmL4ICIiIkMxfBAREZGhGD6IyO9uvvlmXHnllcE+DCIKUQwfREREZCiGDyKqtm+//RadO3dGfHy8Wvp84MCBahnwTz/9FD/88AMsFova5s2bp/bfu3evWi69Zs2aqF27NoYPH45du3adVmMyceJE1K1bF8nJybjrrrtQWFgYxFISkb9F+f0RicgUDh48iBtuuAEvvfQSRowYgaysLCxcuBA33XQT9uzZg8zMTHz88cdqXwkaRUVFGDJkCPr27av2i4qKwjPPPINLLrkE69atQ0xMjNp3zpw5iIuLU4FFgsktt9yigs2zzz4b5BITkb8wfBBRtcNHcXExrrrqKjRr1kzdJrUgQmpCCgoK0KBBA/f+n3/+ORwOBz788ENVGyIknEgtiASNwYMHq9skhHz00Uew2+3o2LEjnnrqKVWb8vTTT8NqZWUtUSTgXzIRVUvXrl0xYMAAFTiuvfZafPDBBzhx4kS5+69duxbbtm1DUlISEhMT1SY1Ivn5+di+fXupx5XgoZOakuzsbNVkQ0SRgTUfRFQtNpsNaWlpWLx4MX755Re89dZbeOyxx7Bs2TKv+0uA6NmzJ7744ovT7pP+HURkHgwfRFRt0nxy7rnnqu3JJ59UzS/Tpk1TTSclJSWl9u3Rowe++uor1KtXT3UkraiGJC8vTzXdiKVLl6paktTU1ICXh4iMwWYXIqoWqeF47rnnsHLlStXB9Pvvv8eRI0fQvn17NG/eXHUi3bx5M44ePao6m44ePRopKSlqhIt0ON25c6fq63Hfffdh37597seVkS1///vfsWnTJsycORP//ve/ce+997K/B1EEYc0HEVWL1F4sWLAAkyZNUiNbpNbj1VdfxdChQ9GrVy8VLORSmlt+++039O/fX+3/8MMPq06qMjqmcePGqt+IZ02I/Ny6dWtccMEFqtOqjKiZMGFCUMtKRP5l0TRN8/NjEhFVi8zzcfLkSUyfPj3Yh0JEAcR6TCIiIjIUwwcREREZis0uREREZCjWfBAREZGhGD6IiIjIUAwfREREZCiGDyIiIjIUwwcREREZiuGDiIiIDMXwQURERIZi+CAiIiIY6f8DpZuBBm0SgkUAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 13
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试集"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:20:28.495660Z",
     "start_time": "2025-01-26T10:20:28.411573Z"
    }
   },
   "source": [
    "model.eval()\n",
    "loss = evaluating(model, val_loader, loss_fct)\n",
    "print(f\"loss:     {loss:.4f}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:     0.4192\n"
     ]
    }
   ],
   "execution_count": 14
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拿到中间输出"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:20:30.038158Z",
     "start_time": "2025-01-26T10:20:30.024134Z"
    }
   },
   "source": [
    "logits, deep_output = model(train_ds[:][0].to(device), return_deep_output=True)"
   ],
   "outputs": [],
   "execution_count": 15
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:20:30.311996Z",
     "start_time": "2025-01-26T10:20:30.307961Z"
    }
   },
   "source": [
    "deep_output.shape"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([11610, 30])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T10:20:30.799341Z",
     "start_time": "2025-01-26T10:20:30.763094Z"
    }
   },
   "source": [
    "# 从这看到deep部分抽取到的特征分布，有些特征没那么重要，或许可以消减一些神经元\n",
    "plt.imshow(deep_output.cpu().detach().numpy().mean(axis=0).reshape(1, -1))\n",
    "plt.yticks([])\n",
    "plt.show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAA8CAYAAAD7aLkGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAACUNJREFUeJzt3WloFFsWwPGbdGdxixqXLGZxiQvjkg9qVByjEIkLiNsHNwYVibjiMi4oaBRk8lAGRBFkYNAvGp4Bo+j7MqIxIiQK+kRkNBjHIclEDcrExDWmu4ZzISH9Xt6Y5JZW+t3/D5rYpqo8nj5ddbrura4Ix3EcBQAArBXpdQAAAMBbNAMAAFiOZgAAAMvRDAAAYDmaAQAALEczAACA5WgGAACwHM0AAACW83dkoWAwqGpra1WfPn1URETEt48KAAAYk+8VbGxsVMnJySoyMtKsGZBGIDU11TwqAADw3VVXV6uUlBSzZkDOCIjJOfuU3x/bpUBi/vGzMubCNyf7+vcz20D/vsYxBPv2NFrf+fmxcQz/+fMUo/WH/PWOcQy+4elG69fNTDCO4da+vxutvyDvT8Yx+EsfGK3/rx8mG8cQV2l2xi/+n5+MY6iZ1cN4G2l/MavLyNgY4xgioqON1v/3pj8Yx5D6g1ke/EPNP/w9XZ9otP7IE1XGMfz3j2lG68fUN5vHMNqsHob8VGu0fnOwSd2s+lvrcdyoGWgZGpBGwB/VtWbAHxGlzLnQDESYvTDKZ76zCBpuw3Ehl76Yrr2Obr6ePsM8+KLN/g8iro/ZtJmuNsdu5jIy1jwGX7RZM+Dv0J7kKzHEdoNcmu4f9P4y2tP3pht58Eea7+dM69Ifaf5adPV41cLvN28GfDHRnr8W4mtD/EwgBADAcjQDAABYjmYAAADL0QwAAGA5mgEAACxHMwAAgOVoBgAAsBzNAAAAlqMZAADAcjQDAABYjmYAAADL0QwAAGA5f0fvhyyam7t+ZzKf86XL67YJxIVNNJltIPDZOIZgwGe0vuNCLgOfze4y1+xCDI5hLgNN5nfKa2gMGq1v8p5oZZjL4CfzGAJNEZ7nIfDJLAYdh2EuIx3zz0cRjrfvTVfen0EX9nOGdSl32zPV/MUsBl+z+Y2KAp8N9zGGr0VLHluO478lwvnaEnJr0ZoalZpqfktLAADw/VVXV6uUlBSzZiAYDKra2lp9P+T2boPY0NCgmwX5x+Li4syjthi5dA+5dAd5dA+5dA+57Bg5xDc2Nqrk5GQVGRlpNkwgG/h/HUULeUF4UdxBLt1DLt1BHt1DLt1DLr+ub9++X12GCYQAAFiOZgAAAMu50gzExMSo/Px8/RNmyKV7yKU7yKN7yKV7yKW7OjSBEAAA/H4xTAAAgOVoBgAAsBzNAAAAlqMZAADAcq40A6dOnVJDhw5VsbGxasqUKeru3btubNYqhw4d0t/u2PYxZswYr8Pq9m7duqUWLFigv11Lcnbp0qWQ38v82IMHD6qkpCTVo0cPNXv2bPX06VPP4g3nXK5Zs+ZXNTp37lzP4u2uCgoK1OTJk/U3tg4ePFgtWrRIVVRUhCzz6dMntXnzZjVgwADVu3dvtXTpUvXq1SvPYg7nXM6aNetXdblhwwbPYra2Gfjxxx/Vzp079SUe9+/fV5mZmWrOnDmqrq7OnQgtMnbsWPXixYvWx+3bt70Oqdt7//69rjlpSNtz9OhRdeLECXX69Gl1584d1atXL12fsjNG53Ip5ODftkYLCwu/a4zhoLS0VB/oy8vL1bVr19SXL19Ubm6uzm+LHTt2qCtXrqiioiK9vHzd+5IlSzyNO1xzKfLy8kLqUt736CTHUFZWlrN58+bW54FAwElOTnYKCgpMN22V/Px8JzMz0+swwpqUc3FxcevzYDDoJCYmOseOHWv9u/r6eicmJsYpLCz0KMrwzKVYvXq1s3DhQs9iCld1dXU6n6Wlpa01GBUV5RQVFbUu8/jxY71MWVmZh5GGXy7FzJkznW3btnka1++B0ZmBpqYmde/ePX3qte19DOR5WVmZyaatJKev5RTt8OHD1apVq1RVVZXXIYW158+fq5cvX4bUp3xHtwxlUZ9dc/PmTX26dvTo0Wrjxo3qzZs3XofU7b19+1b/jI+P1z9lnymfcNvWpQwJpqWlUZedzGWLc+fOqYEDB6px48apffv2qQ8fPngUYfjq0I2Kfsvr169VIBBQCQkJIX8vz588eWIam1XkAHX27Fm9k5XTXIcPH1YzZsxQjx490uNl6DxpBER79dnyO3ScDBHIqexhw4apZ8+eqf3796t58+bpA5jP5/M6vG5J7vi6fft2NX36dH2gElJ70dHRql+/fiHLUpedz6VYuXKlSk9P1x+kHj58qPbu3avnFVy8eNHTeK1qBuAe2am2mDBhgm4OpMAvXLig1q1b52lsgFi+fHnrn8ePH6/rdMSIEfpsQU5OjqexdVcy3i0NPfN/vl0u169fH1KXMllY6lEaVqlPdIzRMIGclpFPBL+cBSvPExMTTTZtPfnUMGrUKFVZWel1KGGrpQapz29DhrNkH0CNtm/Lli3q6tWrqqSkJOQW8FJ7MsRaX18fsjx12flctkc+SAnq8js2A3Kqa+LEier69eshp3Lk+bRp00w2bb13797pzla6XHSNnM6WnWvb+mxoaNBXFVCf5mpqavScAWo0lMy/lINXcXGxunHjhq7DtmSfGRUVFVKXclpb5ghRl53LZXsePHigf1KX33mYQC4rXL16tZo0aZLKyspSx48f15d9rF271nTTVtm1a5e+xluGBuQyI7lUU866rFixwuvQun3T1PYTgEwalJ2BTDCSCVkyxnjkyBE1cuRIvSM5cOCAHluU65XR8VzKQ+axyPXw0mBJo7pnzx6VkZGhL9VE6Ons8+fPq8uXL+v5Pi3zAGTyqnzXhfyUoT/Zd0pe4+Li1NatW3UjMHXqVK/DD6tcSh3K7+fPn6+/s0HmDMhlm9nZ2XoYC53gxiUJJ0+edNLS0pzo6Gh9qWF5ebkbm7XKsmXLnKSkJJ3DIUOG6OeVlZVeh9XtlZSU6EuNfvmQy+BaLi88cOCAk5CQoC8pzMnJcSoqKrwOO+xy+eHDByc3N9cZNGiQviwuPT3dycvLc16+fOl12N1OezmUx5kzZ1qX+fjxo7Np0yanf//+Ts+ePZ3Fixc7L1688DTucMxlVVWVk52d7cTHx+v3d0ZGhrN7927n7du3XocedriFMQAAluPeBAAAWI5mAAAAy9EMAABgOZoBAAAsRzMAAIDlaAYAALAczQAAAJajGQAAwHI0AwAAWI5mAAAAy9EMAABgOZoBAACU3f4HPNIga0EdGA0AAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 17
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": ""
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
